{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generates the table of the ontological issues.\n",
    "\n",
    "### Note this code assumes that you've already computed psx -- the predicted probabilities for all examples in the training set using four-fold cross-validation. If you have no done that you will need to use `imagenet_train_crossval.py` to do this!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These imports enhance Python2/3 compatibility.\n",
    "from __future__ import print_function, absolute_import, division, unicode_literals, with_statement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cleanlab\n",
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "# For visualizing images of label errors\n",
    "from PIL import Image\n",
    "from torchvision import datasets\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# urllib2 for python2 and python3\n",
    "try:\n",
    "    # For Python 3.0 and later\n",
    "    from urllib.request import urlopen\n",
    "except ImportError:\n",
    "    # Fall back to Python 2's urllib2\n",
    "    from urllib2 import urlopen\n",
    "    \n",
    "# where imagenet dataset is located\n",
    "train_dir = '/datasets/datasets/imagenet/val/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set-up name mapping for ImageNet train data\n",
    "url = 'https://gist.githubusercontent.com/aaronpolhamus/964a4411c0906315deb9f4a3723aac57/'\n",
    "url += 'raw/aa66dd9dbf6b56649fa3fab83659b2acbf3cbfd1/map_clsloc.txt'\n",
    "with urlopen(url) as f:\n",
    "    lines = [x.decode('utf-8') for x in f.readlines()]    \n",
    "    nid2name = dict([(l.split(\" \")[0], l.split(\" \")[2][:-1]) for l in lines])\n",
    "    \n",
    "dataset = datasets.ImageFolder(train_dir)\n",
    "nid2idx = dataset.class_to_idx\n",
    "idx2nid = {v: k for k, v in nid2idx.items()}\n",
    "name2nid = {v: k for k, v in nid2name.items()}\n",
    "idx2name = {k: nid2name[v] for k, v in idx2nid.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze the train set on ImageNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CHANGE THIS TO CHANGE EXPERIMENT\n",
    "# pyx_file = 'imagenet_val_out.npy' # NO FINE TUNING\n",
    "pyx_file = 'imagenet__train__model_resnet50__pyx.npy' # trained from scratch with 10fold cv\n",
    "\n",
    "# where imagenet dataset is located\n",
    "train_dir = '/datasets/datasets/imagenet/train/'\n",
    "# Stored results directory\n",
    "pyx_dir = '/datasets/cgn/pyx/imagenet/'\n",
    "\n",
    "# Load in data\n",
    "pyx = np.load(pyx_dir + pyx_file)\n",
    "imgs, labels = [list(z) for  z in zip(*datasets.ImageFolder(train_dir).imgs)]\n",
    "labels = np.array(labels, dtype=int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A bad way to approach this problem might be to just look at the correlation of every column in the probability matrix. The problem is correlation is symmetric and this will correlate everything that has small probabilities. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = np.corrcoef(pyx.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('projectile', 'missile'), ('missile', 'projectile'), ('Eskimo_dog', 'Siberian_husky'), ('Siberian_husky', 'Eskimo_dog'), ('sunglass', 'sunglasses')]\n",
      "['cleaver', 'Australian_terrier', 'miniature_poodle', 'weasel', 'titi', 'ballpoint', 'bow', 'breastplate', 'can_opener', 'CD_player']\n"
     ]
    }
   ],
   "source": [
    "corr_non_diag = corr - np.eye(len(corr)) * corr.diagonal()\n",
    "corr_largest_non_diag_raveled = np.argsort(corr_non_diag.ravel())[::-1]\n",
    "corr_largest_non_diag = np.unravel_index(corr_largest_non_diag_raveled, corr_non_diag.shape)\n",
    "corr_largest_non_diag = list(zip(*(list(z) for z in corr_largest_non_diag)))\n",
    "\n",
    "print([(nid2name[idx2nid[z[0]]], nid2name[idx2nid[z[1]]]) for z in corr_largest_non_diag][:5])\n",
    "print([nid2name[idx2nid[z]] for z in corr.diagonal().argsort()[:10]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Understand ImageNet ontological issues on the TRAIN SET\n",
    "## Uses ARGMAX baseline (not confident learning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "cj = confusion_matrix(np.argmax(pyx, axis=1), labels).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "joint = cleanlab.latent_estimation.estimate_joint(labels, pyx, cj)\n",
    "joint_non_diag = joint - np.eye(len(joint)) * joint.diagonal()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "cj_non_diag = cj - np.eye(len(cj)) * cj.diagonal()\n",
    "largest_non_diag_raveled = np.argsort(cj_non_diag.ravel())[::-1]\n",
    "largest_non_diag = np.unravel_index(largest_non_diag_raveled, cj_non_diag.shape)\n",
    "largest_non_diag = list(zip(*(list(z) for z in largest_non_diag)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checks that joint correctly has rows that are p(s)\n",
    "assert(all(joint.sum(axis = 1) - np.bincount(labels) / len(labels) < 1e-4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index of 'bighorn' in sorted diagonal of cj: 573\n",
      "Index of 'bighorn' in sorted diagonal of joint: 573\n",
      "Index of 'bighorn' in sorted most noisy classes in cj: 237\n",
      "Index of 'bighorn' in sorted most noisy classes in joint: 237\n",
      "Index of 'bighorn' in sorted most noisy true classes in cj: 543\n",
      "Index of 'bighorn' in sorted most noisy true classes in joint: 543\n",
      "Least confident class by diagonal of cj: water_jug 899\n",
      "Least confident class by diagonal of joint: water_jug 899\n",
      "Least confident class by max sum of row of non-diagonal elements of cj: desk 526\n",
      "Least confident class by max sum of column of non-diagonal elements of cj: water_jug 899\n",
      "Largest noise rate: [('projectile', 744), ('missile', 657)]\n"
     ]
    }
   ],
   "source": [
    "class_name = 'bighorn'\n",
    "\n",
    "print(\"Index of '{}' in sorted diagonal of cj: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in cj.diagonal().argsort()].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted diagonal of joint: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in joint.diagonal().argsort()].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted most noisy classes in cj: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in np.argsort(cj_non_diag.sum(axis = 0))[::-1]].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted most noisy classes in joint: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in np.argsort(joint_non_diag.sum(axis = 0))[::-1]].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted most noisy true classes in cj: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in np.argsort(cj_non_diag.sum(axis = 1))[::-1]].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted most noisy true classes in joint: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in np.argsort(joint_non_diag.sum(axis = 1))[::-1]].index(class_name))\n",
    "\n",
    "idx = cj.diagonal().argmin()\n",
    "print(\"Least confident class by diagonal of cj:\", nid2name[idx2nid[idx]], idx)\n",
    "idx = joint.diagonal().argmin()\n",
    "print(\"Least confident class by diagonal of joint:\", nid2name[idx2nid[idx]], idx)\n",
    "idx = cj_non_diag.sum(axis = 0).argmax()\n",
    "print(\"Least confident class by max sum of row of non-diagonal elements of cj:\", nid2name[idx2nid[idx]], idx)\n",
    "idx = joint_non_diag.sum(axis = 1).argmax()\n",
    "print(\"Least confident class by max sum of column of non-diagonal elements of cj:\", nid2name[idx2nid[idx]], idx)\n",
    "print('Largest noise rate:', [(nid2name[idx2nid[z]], z) for z in largest_non_diag[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1107,    5,    1, ...,    0,    0,    0],\n",
       "       [   5, 1146,    0, ...,    0,    0,    0],\n",
       "       [   0,    0,  994, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [   0,    0,    0, ...,  973,    0,    0],\n",
       "       [   0,    0,    0, ...,    0,  716,    0],\n",
       "       [   0,    0,    0, ...,    1,    0,  635]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "edges = [(\n",
    "    idx2name[i].replace('American_chameleon', 'chameleon').replace('typewriter_keyboard', 'keyboard'), \n",
    "    idx2name[j].replace('American_chameleon', 'chameleon').replace('typewriter_keyboard', 'keyboard'), \n",
    "    idx2nid[i], \n",
    "    idx2nid[j], \n",
    "    int(round(cj[i,j])),\n",
    "    joint[i,j].round(6),\n",
    ") for i,j in largest_non_diag[:30]]\n",
    "# nodes = list({z for i,j in largest_non_diag[:30] for z in (idx2name[i], idx2name[j])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(edges)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     False\n",
       "1     False\n",
       "2     False\n",
       "3     False\n",
       "4     False\n",
       "5     False\n",
       "6     False\n",
       "7     False\n",
       "8     False\n",
       "9     False\n",
       "10    False\n",
       "11    False\n",
       "12    False\n",
       "13    False\n",
       "14    False\n",
       "15    False\n",
       "16    False\n",
       "17    False\n",
       "18    False\n",
       "19    False\n",
       "Name: $\\tilde{y}$ name, dtype: bool"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[r\"$\\tilde{y}$ name\"].isin(['a','b'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     494\n",
       "1     400\n",
       "2     398\n",
       "4     369\n",
       "5     362\n",
       "6     362\n",
       "7     338\n",
       "8     336\n",
       "10    333\n",
       "16    293\n",
       "Name: $C(\\tilde{y},y^*)$, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df[r\"$\\tilde{y}$ name\"].isin(['projectile','tub', 'breastplate', 'chameleon', 'green_lizard', 'maillot', 'ram', 'missile', 'corn', 'keyboard'])][r\"$C(\\tilde{y},y^*)$\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{rllllrr}\n",
      "\\toprule\n",
      " Rank & \\$\\textbackslash tilde\\{y\\}\\$ name &                    \\$y\\textasciicircum *\\$ name & \\$\\textbackslash tilde\\{y\\}\\$ nid &  \\$y\\textasciicircum *\\$ nid &  \\$C(\\textbackslash tilde\\{y\\},y\\textasciicircum *)\\$ &  \\$P(\\textbackslash tilde\\{y\\},y\\textasciicircum *)\\$ \\\\\n",
      "\\midrule\n",
      "    1 &       projectile &                       missile &       n04008634 &  n03773504 &                 494 &            0.000386 \\\\\n",
      "    2 &              tub &                       bathtub &       n04493381 &  n02808440 &                 400 &            0.000312 \\\\\n",
      "    3 &      breastplate &                       cuirass &       n02895154 &  n03146219 &                 398 &            0.000311 \\\\\n",
      "    4 &    garden\\_spider &  black\\_and\\_gold\\_garden\\_spider &       n01773797 &  n01773157 &                 377 &            0.000294 \\\\\n",
      "    5 &        chameleon &                  green\\_lizard &       n01682714 &  n01693334 &                 369 &            0.000288 \\\\\n",
      "    6 &     green\\_lizard &                     chameleon &       n01693334 &  n01682714 &                 362 &            0.000283 \\\\\n",
      "    7 &          maillot &                       maillot &       n03710637 &  n03710721 &                 362 &            0.000283 \\\\\n",
      "    8 &              ram &                       bighorn &       n02412080 &  n02415577 &                 338 &            0.000264 \\\\\n",
      "    9 &          missile &                    projectile &       n03773504 &  n04008634 &                 336 &            0.000262 \\\\\n",
      "   10 &     horned\\_viper &                    sidewinder &       n01753488 &  n01756291 &                 335 &            0.000261 \\\\\n",
      "   11 &             corn &                           ear &       n12144580 &  n13133613 &                 333 &            0.000260 \\\\\n",
      "   12 &          cricket &                   grasshopper &       n02229544 &  n02226429 &                 311 &            0.000243 \\\\\n",
      "   13 &    ruffed\\_grouse &                     partridge &       n01797886 &  n01807496 &                 304 &            0.000237 \\\\\n",
      "   14 &              hog &                     wild\\_boar &       n02395406 &  n02396427 &                 299 &            0.000233 \\\\\n",
      "   15 &              ear &                          corn &       n13133613 &  n12144580 &                 298 &            0.000233 \\\\\n",
      "   16 &        tiger\\_cat &                         tabby &       n02123159 &  n02123045 &                 295 &            0.000230 \\\\\n",
      "   17 &         keyboard &                     space\\_bar &       n04505470 &  n04264628 &                 293 &            0.000229 \\\\\n",
      "   18 &         sunglass &                    sunglasses &       n04355933 &  n04356056 &                 287 &            0.000224 \\\\\n",
      "   19 &      mortarboard &                 academic\\_gown &       n03787032 &  n02669723 &                 286 &            0.000223 \\\\\n",
      "   20 &   Siberian\\_husky &                    Eskimo\\_dog &       n02110185 &  n02109961 &                 284 &            0.000222 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_c48caefc_de39_11e9_b1fe_912869241a19row0_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row1_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row2_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row3_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row4_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row5_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row6_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row7_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row8_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row9_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row10_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row11_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row12_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row13_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row14_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row15_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row16_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row17_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row18_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_c48caefc_de39_11e9_b1fe_912869241a19row19_col5 {\n",
       "            width:  50px;\n",
       "        }</style><table id=\"T_c48caefc_de39_11e9_b1fe_912869241a19\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Rank</th>        <th class=\"col_heading level0 col1\" >$\\tilde{y}$ name</th>        <th class=\"col_heading level0 col2\" >$y^*$ name</th>        <th class=\"col_heading level0 col3\" >$\\tilde{y}$ nid</th>        <th class=\"col_heading level0 col4\" >$y^*$ nid</th>        <th class=\"col_heading level0 col5\" >$C(\\tilde{y},y^*)$</th>        <th class=\"col_heading level0 col6\" >$P(\\tilde{y},y^*)$</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row0_col0\" class=\"data row0 col0\" >1</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row0_col1\" class=\"data row0 col1\" >projectile</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row0_col2\" class=\"data row0 col2\" >missile</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row0_col3\" class=\"data row0 col3\" >n04008634</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row0_col4\" class=\"data row0 col4\" >n03773504</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row0_col5\" class=\"data row0 col5\" >494</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row0_col6\" class=\"data row0 col6\" >0.000386</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row1_col0\" class=\"data row1 col0\" >2</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row1_col1\" class=\"data row1 col1\" >tub</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row1_col2\" class=\"data row1 col2\" >bathtub</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row1_col3\" class=\"data row1 col3\" >n04493381</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row1_col4\" class=\"data row1 col4\" >n02808440</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row1_col5\" class=\"data row1 col5\" >400</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row1_col6\" class=\"data row1 col6\" >0.000312</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row2_col0\" class=\"data row2 col0\" >3</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row2_col1\" class=\"data row2 col1\" >breastplate</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row2_col2\" class=\"data row2 col2\" >cuirass</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row2_col3\" class=\"data row2 col3\" >n02895154</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row2_col4\" class=\"data row2 col4\" >n03146219</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row2_col5\" class=\"data row2 col5\" >398</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row2_col6\" class=\"data row2 col6\" >0.000311</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row3_col0\" class=\"data row3 col0\" >4</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row3_col1\" class=\"data row3 col1\" >garden_spider</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row3_col2\" class=\"data row3 col2\" >black_and_gold_garden_spider</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row3_col3\" class=\"data row3 col3\" >n01773797</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row3_col4\" class=\"data row3 col4\" >n01773157</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row3_col5\" class=\"data row3 col5\" >377</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row3_col6\" class=\"data row3 col6\" >0.000294</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row4_col0\" class=\"data row4 col0\" >5</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row4_col1\" class=\"data row4 col1\" >chameleon</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row4_col2\" class=\"data row4 col2\" >green_lizard</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row4_col3\" class=\"data row4 col3\" >n01682714</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row4_col4\" class=\"data row4 col4\" >n01693334</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row4_col5\" class=\"data row4 col5\" >369</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row4_col6\" class=\"data row4 col6\" >0.000288</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row5_col0\" class=\"data row5 col0\" >6</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row5_col1\" class=\"data row5 col1\" >green_lizard</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row5_col2\" class=\"data row5 col2\" >chameleon</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row5_col3\" class=\"data row5 col3\" >n01693334</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row5_col4\" class=\"data row5 col4\" >n01682714</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row5_col5\" class=\"data row5 col5\" >362</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row5_col6\" class=\"data row5 col6\" >0.000283</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row6_col0\" class=\"data row6 col0\" >7</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row6_col1\" class=\"data row6 col1\" >maillot</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row6_col2\" class=\"data row6 col2\" >maillot</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row6_col3\" class=\"data row6 col3\" >n03710637</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row6_col4\" class=\"data row6 col4\" >n03710721</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row6_col5\" class=\"data row6 col5\" >362</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row6_col6\" class=\"data row6 col6\" >0.000283</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row7_col0\" class=\"data row7 col0\" >8</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row7_col1\" class=\"data row7 col1\" >ram</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row7_col2\" class=\"data row7 col2\" >bighorn</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row7_col3\" class=\"data row7 col3\" >n02412080</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row7_col4\" class=\"data row7 col4\" >n02415577</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row7_col5\" class=\"data row7 col5\" >338</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row7_col6\" class=\"data row7 col6\" >0.000264</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row8_col0\" class=\"data row8 col0\" >9</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row8_col1\" class=\"data row8 col1\" >missile</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row8_col2\" class=\"data row8 col2\" >projectile</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row8_col3\" class=\"data row8 col3\" >n03773504</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row8_col4\" class=\"data row8 col4\" >n04008634</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row8_col5\" class=\"data row8 col5\" >336</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row8_col6\" class=\"data row8 col6\" >0.000262</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row9_col0\" class=\"data row9 col0\" >10</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row9_col1\" class=\"data row9 col1\" >horned_viper</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row9_col2\" class=\"data row9 col2\" >sidewinder</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row9_col3\" class=\"data row9 col3\" >n01753488</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row9_col4\" class=\"data row9 col4\" >n01756291</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row9_col5\" class=\"data row9 col5\" >335</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row9_col6\" class=\"data row9 col6\" >0.000261</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row10_col0\" class=\"data row10 col0\" >11</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row10_col1\" class=\"data row10 col1\" >corn</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row10_col2\" class=\"data row10 col2\" >ear</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row10_col3\" class=\"data row10 col3\" >n12144580</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row10_col4\" class=\"data row10 col4\" >n13133613</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row10_col5\" class=\"data row10 col5\" >333</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row10_col6\" class=\"data row10 col6\" >0.00026</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row11_col0\" class=\"data row11 col0\" >12</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row11_col1\" class=\"data row11 col1\" >cricket</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row11_col2\" class=\"data row11 col2\" >grasshopper</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row11_col3\" class=\"data row11 col3\" >n02229544</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row11_col4\" class=\"data row11 col4\" >n02226429</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row11_col5\" class=\"data row11 col5\" >311</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row11_col6\" class=\"data row11 col6\" >0.000243</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row12_col0\" class=\"data row12 col0\" >13</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row12_col1\" class=\"data row12 col1\" >ruffed_grouse</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row12_col2\" class=\"data row12 col2\" >partridge</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row12_col3\" class=\"data row12 col3\" >n01797886</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row12_col4\" class=\"data row12 col4\" >n01807496</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row12_col5\" class=\"data row12 col5\" >304</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row12_col6\" class=\"data row12 col6\" >0.000237</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row13\" class=\"row_heading level0 row13\" >13</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row13_col0\" class=\"data row13 col0\" >14</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row13_col1\" class=\"data row13 col1\" >hog</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row13_col2\" class=\"data row13 col2\" >wild_boar</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row13_col3\" class=\"data row13 col3\" >n02395406</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row13_col4\" class=\"data row13 col4\" >n02396427</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row13_col5\" class=\"data row13 col5\" >299</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row13_col6\" class=\"data row13 col6\" >0.000233</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row14\" class=\"row_heading level0 row14\" >14</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row14_col0\" class=\"data row14 col0\" >15</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row14_col1\" class=\"data row14 col1\" >ear</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row14_col2\" class=\"data row14 col2\" >corn</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row14_col3\" class=\"data row14 col3\" >n13133613</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row14_col4\" class=\"data row14 col4\" >n12144580</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row14_col5\" class=\"data row14 col5\" >298</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row14_col6\" class=\"data row14 col6\" >0.000233</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row15\" class=\"row_heading level0 row15\" >15</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row15_col0\" class=\"data row15 col0\" >16</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row15_col1\" class=\"data row15 col1\" >tiger_cat</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row15_col2\" class=\"data row15 col2\" >tabby</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row15_col3\" class=\"data row15 col3\" >n02123159</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row15_col4\" class=\"data row15 col4\" >n02123045</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row15_col5\" class=\"data row15 col5\" >295</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row15_col6\" class=\"data row15 col6\" >0.00023</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row16\" class=\"row_heading level0 row16\" >16</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row16_col0\" class=\"data row16 col0\" >17</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row16_col1\" class=\"data row16 col1\" >keyboard</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row16_col2\" class=\"data row16 col2\" >space_bar</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row16_col3\" class=\"data row16 col3\" >n04505470</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row16_col4\" class=\"data row16 col4\" >n04264628</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row16_col5\" class=\"data row16 col5\" >293</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row16_col6\" class=\"data row16 col6\" >0.000229</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row17\" class=\"row_heading level0 row17\" >17</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row17_col0\" class=\"data row17 col0\" >18</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row17_col1\" class=\"data row17 col1\" >sunglass</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row17_col2\" class=\"data row17 col2\" >sunglasses</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row17_col3\" class=\"data row17 col3\" >n04355933</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row17_col4\" class=\"data row17 col4\" >n04356056</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row17_col5\" class=\"data row17 col5\" >287</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row17_col6\" class=\"data row17 col6\" >0.000224</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row18\" class=\"row_heading level0 row18\" >18</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row18_col0\" class=\"data row18 col0\" >19</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row18_col1\" class=\"data row18 col1\" >mortarboard</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row18_col2\" class=\"data row18 col2\" >academic_gown</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row18_col3\" class=\"data row18 col3\" >n03787032</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row18_col4\" class=\"data row18 col4\" >n02669723</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row18_col5\" class=\"data row18 col5\" >286</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row18_col6\" class=\"data row18 col6\" >0.000223</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c48caefc_de39_11e9_b1fe_912869241a19level0_row19\" class=\"row_heading level0 row19\" >19</th>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row19_col0\" class=\"data row19 col0\" >20</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row19_col1\" class=\"data row19 col1\" >Siberian_husky</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row19_col2\" class=\"data row19 col2\" >Eskimo_dog</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row19_col3\" class=\"data row19 col3\" >n02110185</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row19_col4\" class=\"data row19 col4\" >n02109961</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row19_col5\" class=\"data row19 col5\" >284</td>\n",
       "                        <td id=\"T_c48caefc_de39_11e9_b1fe_912869241a19row19_col6\" class=\"data row19 col6\" >0.000222</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f259ee889e8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(edges, columns = [r\"$\\tilde{y}$ name\", r\"$y^*$ name\", r\"$\\tilde{y}$ nid\", r\"$y^*$ nid\", r\"$C(\\tilde{y},y^*)$\", r\"$P(\\tilde{y},y^*)$\"])[:20]\n",
    "df.insert(loc = 0, column = 'Rank', value = df.index + 1)\n",
    "tex = df.to_latex(index = False)\n",
    "orig = '\\\\$\\\\textbackslash tilde\\\\{y\\\\}\\\\$ name &    \\\\$y\\\\textasciicircum *\\\\$ name & \\\\$\\\\textbackslash tilde\\\\{y\\\\}\\\\$ nid &  \\\\$y\\\\textasciicircum *\\\\$ nid &  \\\\$C(\\\\textbackslash tilde\\\\{y\\\\},y\\\\textasciicircum *)\\\\$ &  \\\\$P(\\\\textbackslash tilde\\\\{y\\\\},y\\\\textasciicircum *)\\\\$'\n",
    "new = '$\\\\tilde{y}$ name   &   $y^*$ name   &   $\\\\tilde{y}$ nid   &   $y^*$ nid   &   $C(\\\\tilde{y},y^*)$   &   $P(\\\\tilde{y},y^*)$ '\n",
    "tex = tex.replace(orig, new)\n",
    "print(tex)\n",
    "df.style.set_properties(subset=[r\"$C(\\tilde{y},y^*)$\"], **{'width': '50px'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Understand ImageNet ontological issues on the TRAIN SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cj = cleanlab.latent_estimation.compute_confident_joint(labels, pyx)\n",
    "joint = cleanlab.latent_estimation.estimate_joint(labels, pyx, cj)\n",
    "joint_non_diag = joint - np.eye(len(joint)) * joint.diagonal()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cj_non_diag = cj - np.eye(len(cj)) * cj.diagonal()\n",
    "largest_non_diag_raveled = np.argsort(cj_non_diag.ravel())[::-1]\n",
    "largest_non_diag = np.unravel_index(largest_non_diag_raveled, cj_non_diag.shape)\n",
    "largest_non_diag = list(zip(*(list(z) for z in largest_non_diag)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checks that joint correctly has rows that are p(s)\n",
    "assert(all(joint.sum(axis = 1) - np.bincount(labels) / len(labels) < 1e-4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index of 'bighorn' in sorted diagonal of cj: 374\n",
      "Index of 'bighorn' in sorted diagonal of joint: 374\n",
      "Index of 'bighorn' in sorted most noisy classes in cj: 149\n",
      "Index of 'bighorn' in sorted most noisy classes in joint: 150\n",
      "Index of 'bighorn' in sorted most noisy true classes in cj: 325\n",
      "Index of 'bighorn' in sorted most noisy true classes in joint: 325\n",
      "Least confident class by diagonal of cj: English_foxhound 167\n",
      "Least confident class by diagonal of joint: English_foxhound 167\n",
      "Least confident class by max sum of row of non-diagonal elements of cj: maillot 638\n",
      "Least confident class by max sum of column of non-diagonal elements of cj: maillot 638\n",
      "Largest noise rate: [('projectile', 744), ('missile', 657)]\n"
     ]
    }
   ],
   "source": [
    "class_name = 'bighorn'\n",
    "\n",
    "print(\"Index of '{}' in sorted diagonal of cj: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in cj.diagonal().argsort()].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted diagonal of joint: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in joint.diagonal().argsort()].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted most noisy classes in cj: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in np.argsort(cj_non_diag.sum(axis = 0))[::-1]].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted most noisy classes in joint: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in np.argsort(joint_non_diag.sum(axis = 0))[::-1]].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted most noisy true classes in cj: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in np.argsort(cj_non_diag.sum(axis = 1))[::-1]].index(class_name))\n",
    "\n",
    "print(\"Index of '{}' in sorted most noisy true classes in joint: \".format(class_name), end = \"\")\n",
    "print([nid2name[idx2nid[i]] for i in np.argsort(joint_non_diag.sum(axis = 1))[::-1]].index(class_name))\n",
    "\n",
    "idx = cj.diagonal().argmin()\n",
    "print(\"Least confident class by diagonal of cj:\", nid2name[idx2nid[idx]], idx)\n",
    "idx = joint.diagonal().argmin()\n",
    "print(\"Least confident class by diagonal of joint:\", nid2name[idx2nid[idx]], idx)\n",
    "idx = cj_non_diag.sum(axis = 0).argmax()\n",
    "print(\"Least confident class by max sum of row of non-diagonal elements of cj:\", nid2name[idx2nid[idx]], idx)\n",
    "idx = joint_non_diag.sum(axis = 1).argmax()\n",
    "print(\"Least confident class by max sum of column of non-diagonal elements of cj:\", nid2name[idx2nid[idx]], idx)\n",
    "print('Largest noise rate:', [(nid2name[idx2nid[z]], z) for z in largest_non_diag[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1239,    3,    0, ...,    0,    0,    0],\n",
       "       [   2, 1265,    0, ...,    0,    0,    0],\n",
       "       [   0,    0, 1150, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [   0,    0,    0, ..., 1137,    0,    0],\n",
       "       [   0,    0,    0, ...,    0,  874,    0],\n",
       "       [   0,    0,    0, ...,    0,    0,  861]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "edges = [(\n",
    "    idx2name[i].replace('American_chameleon', 'chameleon').replace('typewriter_keyboard', 'keyboard'), \n",
    "    idx2name[j].replace('American_chameleon', 'chameleon').replace('typewriter_keyboard', 'keyboard'), \n",
    "    idx2nid[i], \n",
    "    idx2nid[j], \n",
    "    int(round(cj[i,j])),\n",
    "    joint[i,j].round(6),\n",
    ") for i,j in largest_non_diag[:30]]\n",
    "# nodes = list({z for i,j in largest_non_diag[:30] for z in (idx2name[i], idx2name[j])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'edges' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-cc17b17dd5aa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0medges\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'edges' is not defined"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(edges)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{rllllrr}\n",
      "\\toprule\n",
      " Rank & $\\tilde{y}$ name   &   $y^*$ name   &   $\\tilde{y}$ nid   &   $y^*$ nid   &   $C(\\tilde{y},y^*)$   &   $P(\\tilde{y},y^*)$  \\\\\n",
      "\\midrule\n",
      "    1 &       projectile &       missile &       n04008634 &  n03773504 &                 645 &            0.000503 \\\\\n",
      "    2 &              tub &       bathtub &       n04493381 &  n02808440 &                 539 &            0.000421 \\\\\n",
      "    3 &      breastplate &       cuirass &       n02895154 &  n03146219 &                 476 &            0.000372 \\\\\n",
      "    4 &     green\\_lizard &     chameleon &       n01693334 &  n01682714 &                 437 &            0.000341 \\\\\n",
      "    5 &        chameleon &  green\\_lizard &       n01682714 &  n01693334 &                 435 &            0.000340 \\\\\n",
      "    6 &          missile &    projectile &       n03773504 &  n04008634 &                 433 &            0.000338 \\\\\n",
      "    7 &          maillot &       maillot &       n03710637 &  n03710721 &                 417 &            0.000325 \\\\\n",
      "    8 &     horned\\_viper &    sidewinder &       n01753488 &  n01756291 &                 416 &            0.000325 \\\\\n",
      "    9 &             corn &           ear &       n12144580 &  n13133613 &                 410 &            0.000320 \\\\\n",
      "   10 &         keyboard &     space\\_bar &       n04505470 &  n04264628 &                 407 &            0.000318 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_91c768a6_de37_11e9_b757_65cd1e03546drow0_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_91c768a6_de37_11e9_b757_65cd1e03546drow1_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_91c768a6_de37_11e9_b757_65cd1e03546drow2_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_91c768a6_de37_11e9_b757_65cd1e03546drow3_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_91c768a6_de37_11e9_b757_65cd1e03546drow4_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_91c768a6_de37_11e9_b757_65cd1e03546drow5_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_91c768a6_de37_11e9_b757_65cd1e03546drow6_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_91c768a6_de37_11e9_b757_65cd1e03546drow7_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_91c768a6_de37_11e9_b757_65cd1e03546drow8_col5 {\n",
       "            width:  50px;\n",
       "        }    #T_91c768a6_de37_11e9_b757_65cd1e03546drow9_col5 {\n",
       "            width:  50px;\n",
       "        }</style><table id=\"T_91c768a6_de37_11e9_b757_65cd1e03546d\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Rank</th>        <th class=\"col_heading level0 col1\" >$\\tilde{y}$ name</th>        <th class=\"col_heading level0 col2\" >$y^*$ name</th>        <th class=\"col_heading level0 col3\" >$\\tilde{y}$ nid</th>        <th class=\"col_heading level0 col4\" >$y^*$ nid</th>        <th class=\"col_heading level0 col5\" >$C(\\tilde{y},y^*)$</th>        <th class=\"col_heading level0 col6\" >$P(\\tilde{y},y^*)$</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow0_col0\" class=\"data row0 col0\" >1</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow0_col1\" class=\"data row0 col1\" >projectile</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow0_col2\" class=\"data row0 col2\" >missile</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow0_col3\" class=\"data row0 col3\" >n04008634</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow0_col4\" class=\"data row0 col4\" >n03773504</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow0_col5\" class=\"data row0 col5\" >645</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow0_col6\" class=\"data row0 col6\" >0.000503</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow1_col0\" class=\"data row1 col0\" >2</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow1_col1\" class=\"data row1 col1\" >tub</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow1_col2\" class=\"data row1 col2\" >bathtub</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow1_col3\" class=\"data row1 col3\" >n04493381</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow1_col4\" class=\"data row1 col4\" >n02808440</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow1_col5\" class=\"data row1 col5\" >539</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow1_col6\" class=\"data row1 col6\" >0.000421</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow2_col0\" class=\"data row2 col0\" >3</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow2_col1\" class=\"data row2 col1\" >breastplate</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow2_col2\" class=\"data row2 col2\" >cuirass</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow2_col3\" class=\"data row2 col3\" >n02895154</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow2_col4\" class=\"data row2 col4\" >n03146219</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow2_col5\" class=\"data row2 col5\" >476</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow2_col6\" class=\"data row2 col6\" >0.000372</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow3_col0\" class=\"data row3 col0\" >4</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow3_col1\" class=\"data row3 col1\" >green_lizard</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow3_col2\" class=\"data row3 col2\" >chameleon</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow3_col3\" class=\"data row3 col3\" >n01693334</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow3_col4\" class=\"data row3 col4\" >n01682714</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow3_col5\" class=\"data row3 col5\" >437</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow3_col6\" class=\"data row3 col6\" >0.000341</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow4_col0\" class=\"data row4 col0\" >5</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow4_col1\" class=\"data row4 col1\" >chameleon</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow4_col2\" class=\"data row4 col2\" >green_lizard</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow4_col3\" class=\"data row4 col3\" >n01682714</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow4_col4\" class=\"data row4 col4\" >n01693334</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow4_col5\" class=\"data row4 col5\" >435</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow4_col6\" class=\"data row4 col6\" >0.00034</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow5_col0\" class=\"data row5 col0\" >6</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow5_col1\" class=\"data row5 col1\" >missile</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow5_col2\" class=\"data row5 col2\" >projectile</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow5_col3\" class=\"data row5 col3\" >n03773504</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow5_col4\" class=\"data row5 col4\" >n04008634</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow5_col5\" class=\"data row5 col5\" >433</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow5_col6\" class=\"data row5 col6\" >0.000338</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow6_col0\" class=\"data row6 col0\" >7</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow6_col1\" class=\"data row6 col1\" >maillot</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow6_col2\" class=\"data row6 col2\" >maillot</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow6_col3\" class=\"data row6 col3\" >n03710637</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow6_col4\" class=\"data row6 col4\" >n03710721</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow6_col5\" class=\"data row6 col5\" >417</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow6_col6\" class=\"data row6 col6\" >0.000325</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow7_col0\" class=\"data row7 col0\" >8</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow7_col1\" class=\"data row7 col1\" >horned_viper</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow7_col2\" class=\"data row7 col2\" >sidewinder</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow7_col3\" class=\"data row7 col3\" >n01753488</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow7_col4\" class=\"data row7 col4\" >n01756291</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow7_col5\" class=\"data row7 col5\" >416</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow7_col6\" class=\"data row7 col6\" >0.000325</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow8_col0\" class=\"data row8 col0\" >9</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow8_col1\" class=\"data row8 col1\" >corn</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow8_col2\" class=\"data row8 col2\" >ear</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow8_col3\" class=\"data row8 col3\" >n12144580</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow8_col4\" class=\"data row8 col4\" >n13133613</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow8_col5\" class=\"data row8 col5\" >410</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow8_col6\" class=\"data row8 col6\" >0.00032</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_91c768a6_de37_11e9_b757_65cd1e03546dlevel0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow9_col0\" class=\"data row9 col0\" >10</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow9_col1\" class=\"data row9 col1\" >keyboard</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow9_col2\" class=\"data row9 col2\" >space_bar</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow9_col3\" class=\"data row9 col3\" >n04505470</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow9_col4\" class=\"data row9 col4\" >n04264628</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow9_col5\" class=\"data row9 col5\" >407</td>\n",
       "                        <td id=\"T_91c768a6_de37_11e9_b757_65cd1e03546drow9_col6\" class=\"data row9 col6\" >0.000318</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87f249b6a0>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(edges, columns = [r\"$\\tilde{y}$ name\", r\"$y^*$ name\", r\"$\\tilde{y}$ nid\", r\"$y^*$ nid\", r\"$C(\\tilde{y},y^*)$\", r\"$P(\\tilde{y},y^*)$\"])[:10]\n",
    "df.insert(loc = 0, column = 'Rank', value = df.index + 1)\n",
    "tex = df.to_latex(index = False)\n",
    "orig = '\\\\$\\\\textbackslash tilde\\\\{y\\\\}\\\\$ name &    \\\\$y\\\\textasciicircum *\\\\$ name & \\\\$\\\\textbackslash tilde\\\\{y\\\\}\\\\$ nid &  \\\\$y\\\\textasciicircum *\\\\$ nid &  \\\\$C(\\\\textbackslash tilde\\\\{y\\\\},y\\\\textasciicircum *)\\\\$ &  \\\\$P(\\\\textbackslash tilde\\\\{y\\\\},y\\\\textasciicircum *)\\\\$'\n",
    "new = '$\\\\tilde{y}$ name   &   $y^*$ name   &   $\\\\tilde{y}$ nid   &   $y^*$ nid   &   $C(\\\\tilde{y},y^*)$   &   $P(\\\\tilde{y},y^*)$ '\n",
    "tex = tex.replace(orig, new)\n",
    "print(tex)\n",
    "df.style.set_properties(subset=[r\"$C(\\tilde{y},y^*)$\"], **{'width': '50px'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\\\begin{tabular}{lllllrr}\\n\\\\toprule\\n{} & \\\\$\\\\textbackslash tilde\\\\{y\\\\}\\\\$ name &    \\\\$y\\\\textasciicircum *\\\\$ name & \\\\$\\\\textbackslash tilde\\\\{y\\\\}\\\\$ nid &  \\\\$y\\\\textasciicircum *\\\\$ nid &  \\\\$C(\\\\textbackslash tilde\\\\{y\\\\},y\\\\textasciicircum *)\\\\$ &  \\\\$P(\\\\textbackslash tilde\\\\{y\\\\},y\\\\textasciicircum *)\\\\$ \\\\\\\\\\n\\\\midrule\\n0 &       projectile &       missile &       n04008634 &  n03773504 &                 645 &            0.000503 \\\\\\\\\\n1 &              tub &       bathtub &       n04493381 &  n02808440 &                 539 &            0.000421 \\\\\\\\\\n2 &      breastplate &       cuirass &       n02895154 &  n03146219 &                 476 &            0.000371 \\\\\\\\\\n3 &     green\\\\_lizard &     chameleon &       n01693334 &  n01682714 &                 437 &            0.000341 \\\\\\\\\\n4 &        chameleon &  green\\\\_lizard &       n01682714 &  n01693334 &                 435 &            0.000340 \\\\\\\\\\n5 &          missile &    projectile &       n03773504 &  n04008634 &                 433 &            0.000338 \\\\\\\\\\n6 &          maillot &       maillot &       n03710637 &  n03710721 &                 417 &            0.000326 \\\\\\\\\\n7 &     horned\\\\_viper &    sidewinder &       n01753488 &  n01756291 &                 416 &            0.000324 \\\\\\\\\\n8 &             corn &           ear &       n12144580 &  n13133613 &                 410 &            0.000320 \\\\\\\\\\n9 &         keyboard &     space\\\\_bar &       n04505470 &  n04264628 &                 406 &            0.000317 \\\\\\\\\\n\\\\bottomrule\\n\\\\end{tabular}\\n'"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "645 | n04008634 projectile | n03773504 missile\n",
      "539 | n04493381 tub | n02808440 bathtub\n",
      "476 | n02895154 breastplate | n03146219 cuirass\n",
      "437 | n01693334 green_lizard | n01682714 American_chameleon\n",
      "435 | n01682714 American_chameleon | n01693334 green_lizard\n",
      "433 | n03773504 missile | n04008634 projectile\n",
      "417 | n03710637 maillot | n03710721 maillot\n",
      "416 | n01753488 horned_viper | n01756291 sidewinder\n",
      "410 | n12144580 corn | n13133613 ear\n",
      "406 | n04505470 typewriter_keyboard | n04264628 space_bar\n",
      "399 | n04355933 sunglass | n04356056 sunglasses\n",
      "389 | n02110185 Siberian_husky | n02109961 Eskimo_dog\n",
      "370 | n02979186 cassette_player | n04392985 tape_player\n",
      "369 | n04356056 sunglasses | n04355933 sunglass\n",
      "367 | n02109961 Eskimo_dog | n02110185 Siberian_husky\n",
      "364 | n01797886 ruffed_grouse | n01807496 partridge\n",
      "360 | n02412080 ram | n02415577 bighorn\n",
      "360 | n13133613 ear | n12144580 corn\n",
      "354 | n02395406 hog | n02396427 wild_boar\n",
      "354 | n03642806 laptop | n03832673 notebook\n",
      "351 | n03146219 cuirass | n02895154 breastplate\n",
      "351 | n01773797 garden_spider | n01773157 black_and_gold_garden_spider\n",
      "346 | n02229544 cricket | n02226429 grasshopper\n",
      "342 | n04090263 rifle | n02749479 assault_rifle\n",
      "332 | n03710721 maillot | n03710637 maillot\n",
      "330 | n02123159 tiger_cat | n02123045 tabby\n",
      "326 | n02113624 toy_poodle | n02113712 miniature_poodle\n",
      "317 | n02113712 miniature_poodle | n02113624 toy_poodle\n",
      "314 | n02106030 collie | n02106166 Border_collie\n",
      "302 | n02808440 bathtub | n04493381 tub\n"
     ]
    }
   ],
   "source": [
    "for i,j in largest_non_diag[:30]:\n",
    "    print(int(round(cj[i,j])), \"|\", idx2nid[i], idx2name[i], \"|\",  idx2nid[j], idx2name[j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 30 row sums in confident joint (most noisy classes):\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('n04560804', 'water_jug'),\n",
       " ('n03710637', 'maillot'),\n",
       " ('n04392985', 'tape_player'),\n",
       " ('n09332890', 'lakeside'),\n",
       " ('n03180011', 'desktop_computer'),\n",
       " ('n02808440', 'bathtub'),\n",
       " ('n03832673', 'notebook'),\n",
       " ('n04041544', 'radio'),\n",
       " ('n09428293', 'seashore'),\n",
       " ('n03179701', 'desk'),\n",
       " ('n03773504', 'missile'),\n",
       " ('n02123045', 'tabby'),\n",
       " ('n01756291', 'sidewinder'),\n",
       " ('n04026417', 'purse'),\n",
       " ('n07579787', 'plate'),\n",
       " ('n04356056', 'sunglasses'),\n",
       " ('n02109961', 'Eskimo_dog'),\n",
       " ('n03976657', 'pole'),\n",
       " ('n03782006', 'monitor'),\n",
       " ('n04008634', 'projectile'),\n",
       " ('n03866082', 'overskirt'),\n",
       " ('n03871628', 'packet'),\n",
       " ('n03532672', 'hook'),\n",
       " ('n04152593', 'screen'),\n",
       " ('n01693334', 'green_lizard'),\n",
       " ('n01740131', 'night_snake'),\n",
       " ('n04355933', 'sunglass'),\n",
       " ('n03216828', 'dock'),\n",
       " ('n07930864', 'cup'),\n",
       " ('n02988304', 'CD_player')]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Top 30 row sums in confident joint (most noisy classes):\\n\")\n",
    "[(idx2nid[i], idx2name[i]) for i in np.argsort(cj_non_diag.sum(axis = 0))[::-1][:30]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyze the validation set on ImageNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CHANGE THIS TO CHANGE EXPERIMENT\n",
    "# pyx_file = 'imagenet_val_out.npy' # NO FINE TUNING\n",
    "pyx_file = 'imagenet_val_out_cv_10fold.npy' # fine tuned with 10fold cv\n",
    "\n",
    "# where imagenet dataset is located\n",
    "val_dir = '/datasets/datasets/imagenet/val/'\n",
    "# Stored results directory\n",
    "pyx_dir = '/datasets/cgn/pyx/imagenet/'\n",
    "\n",
    "# Load in data\n",
    "with open(pyx_dir + 'imagenet_val_out_cv_10fold.npy', 'rb') as f:\n",
    "    out = np.load(f)\n",
    "with open(pyx_dir + 'imagenet_val_labels.npy', 'rb') as f:\n",
    "    labels = np.load(f)\n",
    "pyx = torch.nn.functional.softmax(torch.from_numpy(out), dim = 1).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up mapping for imagenet validation data\n",
    "url = 'https://gist.githubusercontent.com/aaronpolhamus/964a4411c0906315deb9f4a3723aac57/'\n",
    "url += 'raw/aa66dd9dbf6b56649fa3fab83659b2acbf3cbfd1/map_clsloc.txt'\n",
    "with urlopen(url) as f:\n",
    "    lines = [x.decode('utf-8') for x in f.readlines()]    \n",
    "    nid2name = dict([(l.split(\" \")[0], l.split(\" \")[2][:-1]) for l in lines])\n",
    "    \n",
    "dataset = datasets.ImageFolder(val_dir)\n",
    "nid2idx = dataset.class_to_idx\n",
    "idx2nid = {v: k for k, v in nid2idx.items()}\n",
    "name2nid = {v: k for k, v in nid2name.items()}\n",
    "idx2name = {k: nid2name[v] for k, v in idx2nid.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cj = cleanlab.latent_estimation.estimate_confident_joint_from_probabilities(labels, pyx)\n",
    "py, nm, inv = cleanlab.latent_estimation.estimate_latent(cj, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cj_non_diag = cj - np.eye(len(cj)) * cj.diagonal()\n",
    "largest_non_diag_raveled = np.argsort(cj_non_diag.ravel())[::-1]\n",
    "largest_non_diag = np.unravel_index(largest_non_diag_raveled, cj_non_diag.shape)\n",
    "largest_non_diag = list(zip(*(list(z) for z in largest_non_diag)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 30 row sums in confident joint (most noisy classes):\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('n02123159', 'tiger_cat'),\n",
       " ('n04355933', 'sunglass'),\n",
       " ('n04152593', 'screen'),\n",
       " ('n02113624', 'toy_poodle'),\n",
       " ('n02979186', 'cassette_player'),\n",
       " ('n02808440', 'bathtub'),\n",
       " ('n03642806', 'laptop'),\n",
       " ('n03637318', 'lampshade'),\n",
       " ('n02988304', 'CD_player'),\n",
       " ('n02107908', 'Appenzeller'),\n",
       " ('n02871525', 'bookshop'),\n",
       " ('n01753488', 'horned_viper'),\n",
       " ('n01682714', 'American_chameleon'),\n",
       " ('n03180011', 'desktop_computer'),\n",
       " ('n02110185', 'Siberian_husky'),\n",
       " ('n02412080', 'ram'),\n",
       " ('n03710637', 'maillot'),\n",
       " ('n12144580', 'corn'),\n",
       " ('n02441942', 'weasel'),\n",
       " ('n07734744', 'mushroom'),\n",
       " ('n03179701', 'desk'),\n",
       " ('n01740131', 'night_snake'),\n",
       " ('n02106030', 'collie'),\n",
       " ('n01667778', 'terrapin'),\n",
       " ('n04008634', 'projectile'),\n",
       " ('n03950228', 'pitcher'),\n",
       " ('n02119022', 'red_fox'),\n",
       " ('n03710721', 'maillot'),\n",
       " ('n03782006', 'monitor'),\n",
       " ('n04560804', 'water_jug')]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Top 30 row sums in confident joint (most noisy classes):\\n\")\n",
    "[(idx2nid[i], idx2name[i]) for i in np.argsort(cj_non_diag.sum(axis = 1))[::-1][:30]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "644.983461962514 | n04008634 projectile | n03773504 missile\n",
      "538.9486260454004 | n04493381 tub | n02808440 bathtub\n",
      "475.6818181818183 | n02895154 breastplate | n03146219 cuirass\n",
      "436.84210526315803 | n01693334 green_lizard | n01682714 American_chameleon\n",
      "435.0343473994112 | n01682714 American_chameleon | n01693334 green_lizard\n",
      "432.89205702647666 | n03773504 missile | n04008634 projectile\n",
      "417.0893054024257 | n03710637 maillot | n03710721 maillot\n",
      "415.62819203268646 | n01753488 horned_viper | n01756291 sidewinder\n",
      "410.09463722397487 | n12144580 corn | n13133613 ear\n",
      "406.25000000000006 | n04505470 typewriter_keyboard | n04264628 space_bar\n",
      "399.1764705882354 | n04355933 sunglass | n04356056 sunglasses\n",
      "389.451476793249 | n02110185 Siberian_husky | n02109961 Eskimo_dog\n",
      "369.8744769874478 | n02979186 cassette_player | n04392985 tape_player\n",
      "368.79432624113485 | n04356056 sunglasses | n04355933 sunglass\n",
      "366.8049792531121 | n02109961 Eskimo_dog | n02110185 Siberian_husky\n",
      "363.6563876651983 | n01797886 ruffed_grouse | n01807496 partridge\n",
      "359.6340150699678 | n02412080 ram | n02415577 bighorn\n",
      "359.51903807615236 | n13133613 ear | n12144580 corn\n",
      "353.98335315101076 | n02395406 hog | n02396427 wild_boar\n",
      "353.96039603960406 | n03642806 laptop | n03832673 notebook\n",
      "351.0659898477158 | n03146219 cuirass | n02895154 breastplate\n",
      "350.64935064935077 | n01773797 garden_spider | n01773157 black_and_gold_garden_spider\n",
      "345.89800443458984 | n02229544 cricket | n02226429 grasshopper\n",
      "341.68704156479225 | n04090263 rifle | n02749479 assault_rifle\n",
      "332.3108384458078 | n03710721 maillot | n03710637 maillot\n",
      "330.1587301587302 | n02123159 tiger_cat | n02123045 tabby\n",
      "326.0869565217391 | n02113624 toy_poodle | n02113712 miniature_poodle\n",
      "316.52173913043487 | n02113712 miniature_poodle | n02113624 toy_poodle\n",
      "313.69565217391306 | n02106030 collie | n02106166 Border_collie\n",
      "301.96292257360966 | n02808440 bathtub | n04493381 tub\n"
     ]
    }
   ],
   "source": [
    "for i,j in largest_non_diag[:30]:\n",
    "    print(cj[i,j], \"|\", idx2nid[i], idx2name[i], \"|\",  idx2nid[j], idx2name[j])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
